﻿/*******************************************************************
Copyright (c) 2011 이재범

*  Permission is hereby granted, free of charge, to any person
*  obtaining a copy of this software and associated documentation
*  files (the "Software"), to deal in the Software without
*  restriction, including without limitation the rights to use,
*  copy, modify, merge, publish, distribute, sublicense, and/or sell
*  copies of the Software, and to permit persons to whom the
*  Software is furnished to do so, subject to the following
*  conditions:
* 
*  The above copyright notice and this permission notice shall be
*  included in all copies or substantial portions of the Software.
* 
*  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
*  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
*  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
*  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
*  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
*  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
*  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
*  OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************/

#include "crawler.h"
#include "common.h"

#include <Windows.h>
#include <boost/filesystem.hpp>

void Crawler::tex_equation_extraction(string &list, string left, string right, vector<string> &result)
{
	string::size_type location = list.find(left);

	while (location != list.npos)
	{
		string::size_type find = location + left.length();
		string::size_type end_find = list.find(right, find);

		if(end_find != list.npos)
		{
			string term = list.substr(find, end_find - find); // 추출된 문장

			if(term.length() >= 25)
			{
				// 여기서 입력받은 문장이 올바른지 확인한다.
				// 여기서 '올바르지 않다' 의 기준은 tex 파일 작성자의 실수로 인하여
				// $ 를 똑바로 처리하지 않았을 때 나타나는 긴 영어 문장을 의미한다.

				string text;
				bool text_start = false;
				double english_word_num = 0; // 문장에서 영어 단어가 차지하는 비율

				for(int find = 0; find < static_cast<int>(term.length()); find ++)
				{
					if(is_alpha(term[find]) && !text_start)
					{
						text_start = true;
						text.push_back(term[find]);
					}
					else if(is_alpha(term[find]) && text_start)
					{
						text.push_back(term[find]);
					}
					else if(text_start && !is_alpha(term[find]))
					{
						// 이 text 가 영어 단어인지 체크한다. 
						if(Equation::word_list.find(text))
						{
							english_word_num += (double)text.length();
						}

						text.clear();
						text_start = false;

						if(term[find] == '\\')
						{
							int search = find + 1;
							for(;search < static_cast<int>(term.length()); search++)
							{
								if(!is_alpha(term[search]) && !is_digit(term[search]))
								{
									find = search;
									find --;
									break;
								}
							}
						}
						continue;
					}
					else if(!text_start && term[find] == '\\')
					{
						int search = find + 1;
						for(;search < static_cast<int>(term.length()); search++)
						{
							if(!is_alpha(term[search]) && !is_digit(term[search]))
							{
								find = search;
								find --;
								break;
							}
						}
						continue;
					}
				}

				// 영어 단어의  수가 월등히 많을 경우
				if(english_word_num > 30)
				{
					//무언가 사람의 실수 인 듯 하다!
					location = end_find; // end_find 를 시작으로 다시 찾는다. 
					continue;
				}
			}
			result.push_back(term);
			location = list.find(left, end_find + 1);
		}
		else
			break;

	}

}
void CrawlerArxiv::operator () ()
{
	cout << "Downloading " << endl;
	download_data = new InternetData(url_to_process, "D:\\equation_search\\database\\dat_" + convert_to_string(crawler_id));

	if(!download_data->is_download_done())
	{
		return;
	}
	// 압축을 해제
	string command = "7z x D:\\equation_search\\database\\dat_"  
		+ convert_to_string(crawler_id)
		+ " -o" 
		+ "D:\\equation_search\\database\\dir_" + convert_to_string(crawler_id);

	system(command.c_str());

	string folder_name = "D:\\equation_search\\database\\dat_" + convert_to_string(crawler_id);
	string directory_name = "D:\\equation_search\\database\\dir_" + convert_to_string(crawler_id);

	boost::filesystem::path tex_file_directory(directory_name);

	if(!boost::filesystem::is_directory(tex_file_directory))
		return;

	boost::filesystem::directory_iterator end_itr;
	vector<string> equation_extracted;

	for(boost::filesystem::directory_iterator dir_itr(tex_file_directory); dir_itr != end_itr; dir_itr ++)
	{
		if(boost::filesystem::is_regular_file(dir_itr->status()))
		{
			string file_name = dir_itr->path().filename().string();

			// tar 파일인 경우
			if(file_name.find(".") != file_name.npos && file_name.substr(file_name.find_last_of(".") + 1) == "tar")
			{
				string tar_command = "7z e D:\\equation_search\\database\\dir_"  
					+ convert_to_string(crawler_id) + "\\" + file_name 
					+ " -oD:\\equation_search\\database\\dir_" + convert_to_string(crawler_id);

				system(tar_command.c_str());
				break;
			}
		}
	}

	for(boost::filesystem::directory_iterator dir_itr(tex_file_directory); dir_itr != end_itr; dir_itr ++)
	{
		if(boost::filesystem::is_regular_file(dir_itr->status()))
		{
			string file_name = dir_itr->path().filename().string();

			// tex 파일인 경우
			if(file_name.find(".") != file_name.npos && file_name.substr(file_name.find_last_of(".") + 1) == "tex")
			{
				Data tex_data(dir_itr->path().string());
				tex_data.load_tex_data();

				tex_equation_extraction(tex_data.get_data(), (string)"$", (string)"$", equation_extracted);
				extract_string_btw(tex_data.get_data(), (string)"\\begin{equation}", (string)"\\end{equation}", equation_extracted);
				extract_string_btw(tex_data.get_data(), (string)"\\begin{equation*}", (string)"\\end{equation*}", equation_extracted);
				extract_string_btw(tex_data.get_data(), (string)"\\[", (string)"\\]", equation_extracted);

				ofstream out;
				out.open("D:\\equation_search\\search\\tex\\original\\eq_" + convert_to_string(crawler_id), ios::app);

				for (int i = 0; i < static_cast<int>(equation_extracted.size());i++)
				{
					out << equation_extracted[i] << endl;
				}

				out.close();

				// hash_file_name 에는 해시 데이터가, eq_file_name 에는 정규화 된 수식이 들어간다. (그냥 eq_file 하고는 다름)
				string hash_file_name = "D:\\equation_search\\search\\tex\\" +  convert_to_string(crawler_id);
				string eq_file_name = "D:\\equation_search\\search\\tex\\normal\\" + convert_to_string(crawler_id);

				// 일단 파일을 생성한다. 
				ofstream create_file;
				create_file.open(hash_file_name);
				create_file.close();

				create_file.open(eq_file_name);
				create_file.close();

				int count = 0;
				for (int i = 0; i < static_cast<int>(equation_extracted.size()); i++)
				{
					hash_data.push_back(new Hash(equation_extracted[i]));

					hash_data[i]->generate_hash_data();

					// hash_file_name 으로 가진 파일에 생성한 해시 데이터를 저장하고
					// eq_file_name 에 정규화 된 수식을 집어넣는다. 
					hash_data[i]->append_hash_data(hash_file_name);
					hash_data[i]->append_equation_data(eq_file_name);

					cout << i << " of " << crawler_id << " : " << hash_data[i]->get_hash() << endl;
				}
				// 속도 향상을 위해 추가한 코드로 hash 데이터를 다 저장한 후 메모리 상에서 지워버린다.
				for(int i = 0; i < static_cast<int>(hash_data.size()); i++)
				{
					delete hash_data[i];
				}

				hash_data.clear(); 
				equation_extracted.clear(); // 중복해서 수식 쓰는 것 방지

				ofstream result("D:\\equation_search\\search\\tex\\result");
				result << crawler_id << endl;
				result.close();
			}
		}
	}
	delete download_data;
}
void CrawlerArxiv::start()
{
	basic_string<TCHAR> site_url = L"http://arxiv.org/e-print/";
	int num = 1001;

	//for(; num <= 1112; num += 100)
	//{
	//for(; (num % 100) <= 12; num += 10)
	//	{
	for(int i =total_created_crawler_number + 1000; i < 3000; i++)
	{
		basic_string<TCHAR> url = site_url + string_to_wstring_conv(convert_to_string(num) + "." + convert_to_string(i));
		CrawlerArxiv *c = new CrawlerArxiv (url);
		c->operator()();
		delete c;
	}
	//	}

	//	num = num - 12;
	//}
}
void CrawlerArxiv::restart()
{
	basic_string<TCHAR> site_url = L"http://arxiv.org/e-print/";
	ifstream in("D:\\equation_search\\search\\tex\\result");

	in >> total_created_crawler_number;
	total_created_crawler_number ++;

	in.close();

	boost::filesystem::path tex_file_directory("D:\\equation_search\\search\\tex");
	boost::filesystem::directory_iterator end_itr;

	for(boost::filesystem::directory_iterator dir_itr(tex_file_directory); dir_itr != end_itr; dir_itr ++)
	{
		if(boost::filesystem::is_regular_file(dir_itr->status()))
		{
			string file_name = dir_itr->path().filename().string();

			int file_num = -1;
			istringstream ( file_name ) >> file_num;

			if(file_num >= total_created_crawler_number )
			{
				boost::filesystem::remove(dir_itr->path());
			}

		}
	}

	boost::filesystem::path eq_file_directory("D:\\equation_search\\search\\tex\\original");
	for(boost::filesystem::directory_iterator dir_itr(eq_file_directory); dir_itr != end_itr; dir_itr ++)
	{
		if(boost::filesystem::is_regular_file(dir_itr->status()))
		{
			string file_name = dir_itr->path().filename().string();
			file_name.erase(0, 3); // 처음의 eq_ 를 지움

			int file_num = -1;
			istringstream ( file_name ) >> file_num;

			if(file_num >= total_created_crawler_number )
			{
				boost::filesystem::remove(dir_itr->path());
			}
		}
	}

	boost::filesystem::path normal_file_directory("D:\\equation_search\\search\\tex\\normal");
	for(boost::filesystem::directory_iterator dir_itr(normal_file_directory); dir_itr != end_itr; dir_itr ++)
	{
		if(boost::filesystem::is_regular_file(dir_itr->status()))
		{
			string file_name = dir_itr->path().filename().string();

			int file_num = -1;
			istringstream ( file_name ) >> file_num;

			if(file_num >= total_created_crawler_number )
			{
				boost::filesystem::remove(dir_itr->path());
			}
		}
	}
	start();
}